{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['cifar-10-python.tar.gz']\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"../input\"))\n",
    "\n",
    "import time\n",
    "\n",
    "# import pytorch\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.optim import SGD,Adam,lr_scheduler\n",
    "from torch.utils.data import random_split\n",
    "import torchvision\n",
    "from torchvision import transforms, datasets\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define transformations for train\n",
    "train_transform = transforms.Compose([\n",
    "    transforms.RandomHorizontalFlip(p=.40),\n",
    "    transforms.RandomRotation(30),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define transformations for test\n",
    "test_transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define training dataloader\n",
    "def get_training_dataloader(train_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        train_transform: transfroms for train dataset\n",
    "        path: path to cifar100 training python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: train_data_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_train = train_transform\n",
    "    cifar10_training = torchvision.datasets.CIFAR10(root='.', train=True, download=True, transform=transform_train)\n",
    "    cifar10_training_loader = DataLoader(\n",
    "        cifar10_training, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_training_loader\n",
    "\n",
    "# define test dataloader\n",
    "def get_testing_dataloader(test_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        test_transform: transforms for test dataset\n",
    "        path: path to cifar100 test python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: cifar100_test_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_test = test_transform\n",
    "    cifar10_test = torchvision.datasets.CIFAR10(root='.', train=False, download=True, transform=transform_test)\n",
    "    cifar10_test_loader = DataLoader(\n",
    "        cifar10_test, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_test_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement mish activation function\n",
    "def f_mish(input):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "    '''\n",
    "    return input * torch.tanh(F.softplus(input))\n",
    "\n",
    "# implement class wrapper for mish activation function\n",
    "class mish(nn.Module):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = mish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_mish(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement swish activation function\n",
    "def f_swish(input):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "    '''\n",
    "    return input * torch.sigmoid(input)\n",
    "\n",
    "# implement class wrapper for swish activation function\n",
    "class swish(nn.Module):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = swish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_swish(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BasicResidualSEBlock(nn.Module):\n",
    "\n",
    "    expansion = 1\n",
    "\n",
    "    def __init__(self, in_channels, out_channels, stride, r=16, activation = 'relu'):\n",
    "        super().__init__()\n",
    "        \n",
    "        if activation == 'relu':\n",
    "            f_activation = nn.ReLU(inplace=True)\n",
    "            self.activation = F.relu\n",
    "            \n",
    "        if activation == 'swish':\n",
    "            f_activation = swish()\n",
    "            self.activation = f_swish\n",
    "            \n",
    "        if activation == 'mish':\n",
    "            f_activation = mish()\n",
    "            self.activation = f_mish\n",
    "\n",
    "        self.residual = nn.Sequential(\n",
    "            nn.Conv2d(in_channels, out_channels, 3, stride=stride, padding=1),\n",
    "            nn.BatchNorm2d(out_channels),\n",
    "            f_activation,\n",
    "            \n",
    "            nn.Conv2d(out_channels, out_channels * self.expansion, 3, padding=1),\n",
    "            nn.BatchNorm2d(out_channels * self.expansion),\n",
    "            f_activation\n",
    "        )\n",
    "\n",
    "        self.shortcut = nn.Sequential()\n",
    "        if stride != 1 or in_channels != out_channels * self.expansion:\n",
    "            self.shortcut = nn.Sequential(\n",
    "                nn.Conv2d(in_channels, out_channels * self.expansion, 1, stride=stride),\n",
    "                nn.BatchNorm2d(out_channels * self.expansion)\n",
    "            )\n",
    "        \n",
    "        self.squeeze = nn.AdaptiveAvgPool2d(1)\n",
    "        self.excitation = nn.Sequential(\n",
    "            nn.Linear(out_channels * self.expansion, out_channels * self.expansion // r),\n",
    "            f_activation,\n",
    "            nn.Linear(out_channels * self.expansion // r, out_channels * self.expansion),\n",
    "            nn.Sigmoid()\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        shortcut = self.shortcut(x)\n",
    "        residual = self.residual(x)\n",
    "\n",
    "        squeeze = self.squeeze(residual)\n",
    "        squeeze = squeeze.view(squeeze.size(0), -1)\n",
    "        excitation = self.excitation(squeeze)\n",
    "        excitation = excitation.view(residual.size(0), residual.size(1), 1, 1)\n",
    "\n",
    "        x = residual * excitation.expand_as(residual) + shortcut\n",
    "\n",
    "        return self.activation(x)\n",
    "\n",
    "class BottleneckResidualSEBlock(nn.Module):\n",
    "\n",
    "    expansion = 4\n",
    "\n",
    "    def __init__(self, in_channels, out_channels, stride, r=16, activation = 'relu'):\n",
    "        super().__init__()\n",
    "        \n",
    "        if activation == 'relu':\n",
    "            f_activation = nn.ReLU(inplace=True)\n",
    "            self.activation = F.relu\n",
    "            \n",
    "        if activation == 'swish':\n",
    "            f_activation = swish()\n",
    "            self.activation = f_swish\n",
    "            \n",
    "        if activation == 'mish':\n",
    "            f_activation = mish()\n",
    "            self.activation = f_mish\n",
    "\n",
    "        self.residual = nn.Sequential(\n",
    "            nn.Conv2d(in_channels, out_channels, 1),\n",
    "            nn.BatchNorm2d(out_channels),\n",
    "            f_activation,\n",
    "\n",
    "            nn.Conv2d(out_channels, out_channels, 3, stride=stride, padding=1),\n",
    "            nn.BatchNorm2d(out_channels),\n",
    "            f_activation,\n",
    "\n",
    "            nn.Conv2d(out_channels, out_channels * self.expansion, 1),\n",
    "            nn.BatchNorm2d(out_channels * self.expansion),\n",
    "            f_activation\n",
    "        )\n",
    "\n",
    "        self.squeeze = nn.AdaptiveAvgPool2d(1)\n",
    "        self.excitation = nn.Sequential(\n",
    "            nn.Linear(out_channels * self.expansion, out_channels * self.expansion // r),\n",
    "            f_activation,\n",
    "            nn.Linear(out_channels * self.expansion // r, out_channels * self.expansion),\n",
    "            nn.Sigmoid()\n",
    "        )\n",
    "\n",
    "        self.shortcut = nn.Sequential()\n",
    "        if stride != 1 or in_channels != out_channels * self.expansion:\n",
    "            self.shortcut = nn.Sequential(\n",
    "                nn.Conv2d(in_channels, out_channels * self.expansion, 1, stride=stride),\n",
    "                nn.BatchNorm2d(out_channels * self.expansion)\n",
    "            )\n",
    "\n",
    "    def forward(self, x):\n",
    "\n",
    "        shortcut = self.shortcut(x)\n",
    "\n",
    "        residual = self.residual(x)\n",
    "        squeeze = self.squeeze(residual)\n",
    "        squeeze = squeeze.view(squeeze.size(0), -1)\n",
    "        excitation = self.excitation(squeeze)\n",
    "        excitation = excitation.view(residual.size(0), residual.size(1), 1, 1)\n",
    "\n",
    "        x = residual * excitation.expand_as(residual) + shortcut\n",
    "\n",
    "        return self.activation(x)\n",
    "\n",
    "class SEResNet(nn.Module):\n",
    "\n",
    "    def __init__(self, block, block_num, class_num=10, activation = 'relu'):\n",
    "        super().__init__()\n",
    "\n",
    "        self.in_channels = 64\n",
    "        \n",
    "        if activation == 'relu':\n",
    "            f_activation = nn.ReLU(inplace=True)\n",
    "            self.activation = F.relu\n",
    "            \n",
    "        if activation == 'swish':\n",
    "            f_activation = swish()\n",
    "            self.activation = f_swish\n",
    "            \n",
    "        if activation == 'mish':\n",
    "            f_activation = mish()\n",
    "            self.activation = f_mish\n",
    "\n",
    "        self.pre = nn.Sequential(\n",
    "            nn.Conv2d(3, 64, 3, padding=1),\n",
    "            nn.BatchNorm2d(64),\n",
    "            f_activation\n",
    "        )\n",
    "\n",
    "        self.stage1 = self._make_stage(block, block_num[0], 64, 1, activation = activation)\n",
    "        self.stage2 = self._make_stage(block, block_num[1], 128, 2, activation = activation)\n",
    "        self.stage3 = self._make_stage(block, block_num[2], 256, 2, activation = activation)\n",
    "        self.stage4 = self._make_stage(block, block_num[3], 516, 2, activation = activation)\n",
    "\n",
    "        self.linear = nn.Linear(self.in_channels, class_num)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = self.pre(x)\n",
    "\n",
    "        x = self.stage1(x)\n",
    "        x = self.stage2(x)\n",
    "        x = self.stage3(x)\n",
    "        x = self.stage4(x)\n",
    "\n",
    "        x = F.adaptive_avg_pool2d(x, 1)\n",
    "        x = x.view(x.size(0), -1)\n",
    "\n",
    "        x = self.linear(x)\n",
    "\n",
    "        return x\n",
    "\n",
    "    \n",
    "    def _make_stage(self, block, num, out_channels, stride, activation = 'relu'):\n",
    "\n",
    "        layers = []\n",
    "        layers.append(block(self.in_channels, out_channels, stride, activation = activation))\n",
    "        self.in_channels = out_channels * block.expansion\n",
    "\n",
    "        while num - 1:\n",
    "            layers.append(block(self.in_channels, out_channels, 1, activation = activation))\n",
    "            num -= 1\n",
    "        \n",
    "        return nn.Sequential(*layers)\n",
    "        \n",
    "def seresnet18(activation = 'relu'):\n",
    "    return SEResNet(BasicResidualSEBlock, [2, 2, 2, 2], activation = activation)\n",
    "\n",
    "def seresnet34(activation = 'relu'):\n",
    "    return SEResNet(BasicResidualSEBlock, [3, 4, 6, 3], activation = activation)\n",
    "\n",
    "def seresnet50(activation = 'relu'):\n",
    "    return SEResNet(BottleneckResidualSEBlock, [3, 4, 6, 3], activation = activation)\n",
    "\n",
    "def seresnet101(activation = 'relu'):\n",
    "    return SEResNet(BottleneckResidualSEBlock, [3, 4, 23, 3], activation = activation)\n",
    "\n",
    "def seresnet152(activation = 'relu'):\n",
    "    return SEResNet(BottleneckResidualSEBlock, [3, 8, 36, 3], activation = activation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./cifar-10-python.tar.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "170500096it [00:06, 27481499.25it/s]                               \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "trainloader = get_training_dataloader(train_transform)\n",
    "testloader = get_testing_dataloader(test_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "epochs = 100\n",
    "batch_size = 128\n",
    "learning_rate = 0.001\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = seresnet18(activation = 'mish')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set loss function\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# set optimizer, only train the classifier parameters, feature parameters are frozen\n",
    "optimizer = Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats = pd.DataFrame(columns = ['Epoch', 'Time per epoch', 'Avg time per step', 'Train loss', 'Train accuracy', 'Train top-3 accuracy','Test loss', 'Test accuracy', 'Test top-3 accuracy']) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100.. Time per epoch: 59.5058.. Average time per step: 0.1522.. Train loss: 1.4410.. Train accuracy: 0.4719.. Top-3 train accuracy: 0.7975.. Test loss: 1.1111.. Test accuracy: 0.5979.. Top-3 test accuracy: 0.8827\n",
      "Epoch 2/100.. Time per epoch: 58.5211.. Average time per step: 0.1497.. Train loss: 1.0474.. Train accuracy: 0.6266.. Top-3 train accuracy: 0.8901.. Test loss: 0.8924.. Test accuracy: 0.6863.. Top-3 test accuracy: 0.9165\n",
      "Epoch 3/100.. Time per epoch: 58.5008.. Average time per step: 0.1496.. Train loss: 0.8766.. Train accuracy: 0.6898.. Top-3 train accuracy: 0.9190.. Test loss: 0.7806.. Test accuracy: 0.7277.. Top-3 test accuracy: 0.9346\n",
      "Epoch 4/100.. Time per epoch: 58.4295.. Average time per step: 0.1494.. Train loss: 0.7576.. Train accuracy: 0.7317.. Top-3 train accuracy: 0.9367.. Test loss: 0.6503.. Test accuracy: 0.7715.. Top-3 test accuracy: 0.9529\n",
      "Epoch 5/100.. Time per epoch: 58.4691.. Average time per step: 0.1495.. Train loss: 0.6693.. Train accuracy: 0.7663.. Top-3 train accuracy: 0.9485.. Test loss: 0.5966.. Test accuracy: 0.7917.. Top-3 test accuracy: 0.9598\n",
      "Epoch 6/100.. Time per epoch: 58.0231.. Average time per step: 0.1484.. Train loss: 0.5977.. Train accuracy: 0.7916.. Top-3 train accuracy: 0.9565.. Test loss: 0.5645.. Test accuracy: 0.8021.. Top-3 test accuracy: 0.9642\n",
      "Epoch 7/100.. Time per epoch: 58.0686.. Average time per step: 0.1485.. Train loss: 0.5437.. Train accuracy: 0.8090.. Top-3 train accuracy: 0.9631.. Test loss: 0.4856.. Test accuracy: 0.8345.. Top-3 test accuracy: 0.9702\n",
      "Epoch 8/100.. Time per epoch: 58.1408.. Average time per step: 0.1487.. Train loss: 0.5034.. Train accuracy: 0.8250.. Top-3 train accuracy: 0.9669.. Test loss: 0.4975.. Test accuracy: 0.8338.. Top-3 test accuracy: 0.9693\n",
      "Epoch 9/100.. Time per epoch: 58.0765.. Average time per step: 0.1485.. Train loss: 0.4637.. Train accuracy: 0.8369.. Top-3 train accuracy: 0.9714.. Test loss: 0.4508.. Test accuracy: 0.8477.. Top-3 test accuracy: 0.9737\n",
      "Epoch 10/100.. Time per epoch: 58.1137.. Average time per step: 0.1486.. Train loss: 0.4342.. Train accuracy: 0.8477.. Top-3 train accuracy: 0.9747.. Test loss: 0.4432.. Test accuracy: 0.8466.. Top-3 test accuracy: 0.9739\n",
      "Epoch 11/100.. Time per epoch: 58.1785.. Average time per step: 0.1488.. Train loss: 0.4027.. Train accuracy: 0.8582.. Top-3 train accuracy: 0.9769.. Test loss: 0.4274.. Test accuracy: 0.8584.. Top-3 test accuracy: 0.9770\n",
      "Epoch 12/100.. Time per epoch: 58.2584.. Average time per step: 0.1490.. Train loss: 0.3746.. Train accuracy: 0.8684.. Top-3 train accuracy: 0.9801.. Test loss: 0.4125.. Test accuracy: 0.8625.. Top-3 test accuracy: 0.9777\n",
      "Epoch 13/100.. Time per epoch: 58.2688.. Average time per step: 0.1490.. Train loss: 0.3531.. Train accuracy: 0.8751.. Top-3 train accuracy: 0.9819.. Test loss: 0.4043.. Test accuracy: 0.8666.. Top-3 test accuracy: 0.9798\n",
      "Epoch 14/100.. Time per epoch: 58.3006.. Average time per step: 0.1491.. Train loss: 0.3286.. Train accuracy: 0.8850.. Top-3 train accuracy: 0.9837.. Test loss: 0.3827.. Test accuracy: 0.8715.. Top-3 test accuracy: 0.9799\n",
      "Epoch 15/100.. Time per epoch: 58.2233.. Average time per step: 0.1489.. Train loss: 0.3078.. Train accuracy: 0.8920.. Top-3 train accuracy: 0.9854.. Test loss: 0.4396.. Test accuracy: 0.8582.. Top-3 test accuracy: 0.9757\n",
      "Epoch 16/100.. Time per epoch: 58.0980.. Average time per step: 0.1486.. Train loss: 0.2919.. Train accuracy: 0.8972.. Top-3 train accuracy: 0.9872.. Test loss: 0.3837.. Test accuracy: 0.8750.. Top-3 test accuracy: 0.9812\n",
      "Epoch 17/100.. Time per epoch: 58.2424.. Average time per step: 0.1490.. Train loss: 0.2716.. Train accuracy: 0.9031.. Top-3 train accuracy: 0.9888.. Test loss: 0.3785.. Test accuracy: 0.8745.. Top-3 test accuracy: 0.9818\n",
      "Epoch 18/100.. Time per epoch: 58.3789.. Average time per step: 0.1493.. Train loss: 0.2576.. Train accuracy: 0.9101.. Top-3 train accuracy: 0.9896.. Test loss: 0.3831.. Test accuracy: 0.8791.. Top-3 test accuracy: 0.9806\n",
      "Epoch 19/100.. Time per epoch: 58.2301.. Average time per step: 0.1489.. Train loss: 0.2410.. Train accuracy: 0.9148.. Top-3 train accuracy: 0.9913.. Test loss: 0.3662.. Test accuracy: 0.8826.. Top-3 test accuracy: 0.9820\n",
      "Epoch 20/100.. Time per epoch: 58.4980.. Average time per step: 0.1496.. Train loss: 0.2275.. Train accuracy: 0.9191.. Top-3 train accuracy: 0.9917.. Test loss: 0.3608.. Test accuracy: 0.8839.. Top-3 test accuracy: 0.9832\n",
      "Epoch 21/100.. Time per epoch: 58.4391.. Average time per step: 0.1495.. Train loss: 0.2108.. Train accuracy: 0.9258.. Top-3 train accuracy: 0.9935.. Test loss: 0.3808.. Test accuracy: 0.8841.. Top-3 test accuracy: 0.9826\n",
      "Epoch 22/100.. Time per epoch: 58.6067.. Average time per step: 0.1499.. Train loss: 0.2002.. Train accuracy: 0.9291.. Top-3 train accuracy: 0.9934.. Test loss: 0.3754.. Test accuracy: 0.8851.. Top-3 test accuracy: 0.9840\n",
      "Epoch 23/100.. Time per epoch: 58.3566.. Average time per step: 0.1492.. Train loss: 0.1872.. Train accuracy: 0.9344.. Top-3 train accuracy: 0.9942.. Test loss: 0.3650.. Test accuracy: 0.8903.. Top-3 test accuracy: 0.9841\n",
      "Epoch 24/100.. Time per epoch: 58.4070.. Average time per step: 0.1494.. Train loss: 0.1795.. Train accuracy: 0.9364.. Top-3 train accuracy: 0.9946.. Test loss: 0.3997.. Test accuracy: 0.8849.. Top-3 test accuracy: 0.9781\n",
      "Epoch 25/100.. Time per epoch: 57.8835.. Average time per step: 0.1480.. Train loss: 0.1687.. Train accuracy: 0.9405.. Top-3 train accuracy: 0.9954.. Test loss: 0.3901.. Test accuracy: 0.8852.. Top-3 test accuracy: 0.9830\n",
      "Epoch 26/100.. Time per epoch: 58.0625.. Average time per step: 0.1485.. Train loss: 0.1586.. Train accuracy: 0.9443.. Top-3 train accuracy: 0.9957.. Test loss: 0.4094.. Test accuracy: 0.8796.. Top-3 test accuracy: 0.9830\n",
      "Epoch 27/100.. Time per epoch: 58.2370.. Average time per step: 0.1489.. Train loss: 0.1524.. Train accuracy: 0.9470.. Top-3 train accuracy: 0.9957.. Test loss: 0.3868.. Test accuracy: 0.8846.. Top-3 test accuracy: 0.9838\n",
      "Epoch 28/100.. Time per epoch: 57.9287.. Average time per step: 0.1482.. Train loss: 0.1429.. Train accuracy: 0.9490.. Top-3 train accuracy: 0.9964.. Test loss: 0.3967.. Test accuracy: 0.8890.. Top-3 test accuracy: 0.9826\n",
      "Epoch 29/100.. Time per epoch: 57.8662.. Average time per step: 0.1480.. Train loss: 0.1375.. Train accuracy: 0.9510.. Top-3 train accuracy: 0.9971.. Test loss: 0.3824.. Test accuracy: 0.8901.. Top-3 test accuracy: 0.9856\n",
      "Epoch 30/100.. Time per epoch: 57.8405.. Average time per step: 0.1479.. Train loss: 0.1247.. Train accuracy: 0.9558.. Top-3 train accuracy: 0.9971.. Test loss: 0.3922.. Test accuracy: 0.8924.. Top-3 test accuracy: 0.9844\n",
      "Epoch 31/100.. Time per epoch: 58.0472.. Average time per step: 0.1485.. Train loss: 0.1211.. Train accuracy: 0.9567.. Top-3 train accuracy: 0.9976.. Test loss: 0.4066.. Test accuracy: 0.8900.. Top-3 test accuracy: 0.9833\n",
      "Epoch 32/100.. Time per epoch: 58.0543.. Average time per step: 0.1485.. Train loss: 0.1149.. Train accuracy: 0.9600.. Top-3 train accuracy: 0.9974.. Test loss: 0.3933.. Test accuracy: 0.8939.. Top-3 test accuracy: 0.9845\n",
      "Epoch 33/100.. Time per epoch: 57.9562.. Average time per step: 0.1482.. Train loss: 0.1116.. Train accuracy: 0.9607.. Top-3 train accuracy: 0.9980.. Test loss: 0.3912.. Test accuracy: 0.8937.. Top-3 test accuracy: 0.9850\n",
      "Epoch 34/100.. Time per epoch: 57.9038.. Average time per step: 0.1481.. Train loss: 0.1007.. Train accuracy: 0.9650.. Top-3 train accuracy: 0.9981.. Test loss: 0.4046.. Test accuracy: 0.8919.. Top-3 test accuracy: 0.9843\n",
      "Epoch 35/100.. Time per epoch: 58.0285.. Average time per step: 0.1484.. Train loss: 0.1025.. Train accuracy: 0.9644.. Top-3 train accuracy: 0.9982.. Test loss: 0.4032.. Test accuracy: 0.8952.. Top-3 test accuracy: 0.9846\n",
      "Epoch 36/100.. Time per epoch: 58.0629.. Average time per step: 0.1485.. Train loss: 0.0977.. Train accuracy: 0.9651.. Top-3 train accuracy: 0.9986.. Test loss: 0.4092.. Test accuracy: 0.8944.. Top-3 test accuracy: 0.9836\n",
      "Epoch 37/100.. Time per epoch: 58.1191.. Average time per step: 0.1486.. Train loss: 0.0890.. Train accuracy: 0.9687.. Top-3 train accuracy: 0.9987.. Test loss: 0.4160.. Test accuracy: 0.8934.. Top-3 test accuracy: 0.9836\n",
      "Epoch 38/100.. Time per epoch: 57.9099.. Average time per step: 0.1481.. Train loss: 0.0885.. Train accuracy: 0.9680.. Top-3 train accuracy: 0.9987.. Test loss: 0.4340.. Test accuracy: 0.8904.. Top-3 test accuracy: 0.9847\n",
      "Epoch 39/100.. Time per epoch: 57.9002.. Average time per step: 0.1481.. Train loss: 0.0854.. Train accuracy: 0.9699.. Top-3 train accuracy: 0.9988.. Test loss: 0.4269.. Test accuracy: 0.8930.. Top-3 test accuracy: 0.9844\n",
      "Epoch 40/100.. Time per epoch: 57.7297.. Average time per step: 0.1476.. Train loss: 0.0851.. Train accuracy: 0.9698.. Top-3 train accuracy: 0.9990.. Test loss: 0.4152.. Test accuracy: 0.8968.. Top-3 test accuracy: 0.9842\n",
      "Epoch 41/100.. Time per epoch: 57.8792.. Average time per step: 0.1480.. Train loss: 0.0754.. Train accuracy: 0.9739.. Top-3 train accuracy: 0.9989.. Test loss: 0.4398.. Test accuracy: 0.8941.. Top-3 test accuracy: 0.9831\n",
      "Epoch 42/100.. Time per epoch: 57.6727.. Average time per step: 0.1475.. Train loss: 0.0767.. Train accuracy: 0.9727.. Top-3 train accuracy: 0.9989.. Test loss: 0.4618.. Test accuracy: 0.8874.. Top-3 test accuracy: 0.9828\n",
      "Epoch 43/100.. Time per epoch: 57.7151.. Average time per step: 0.1476.. Train loss: 0.0712.. Train accuracy: 0.9753.. Top-3 train accuracy: 0.9991.. Test loss: 0.4396.. Test accuracy: 0.8888.. Top-3 test accuracy: 0.9845\n",
      "Epoch 44/100.. Time per epoch: 57.8193.. Average time per step: 0.1479.. Train loss: 0.0692.. Train accuracy: 0.9754.. Top-3 train accuracy: 0.9992.. Test loss: 0.4268.. Test accuracy: 0.8986.. Top-3 test accuracy: 0.9850\n",
      "Epoch 45/100.. Time per epoch: 58.1237.. Average time per step: 0.1487.. Train loss: 0.0676.. Train accuracy: 0.9762.. Top-3 train accuracy: 0.9992.. Test loss: 0.4514.. Test accuracy: 0.8924.. Top-3 test accuracy: 0.9833\n",
      "Epoch 46/100.. Time per epoch: 58.0621.. Average time per step: 0.1485.. Train loss: 0.0666.. Train accuracy: 0.9770.. Top-3 train accuracy: 0.9991.. Test loss: 0.4390.. Test accuracy: 0.8951.. Top-3 test accuracy: 0.9843\n",
      "Epoch 47/100.. Time per epoch: 57.9843.. Average time per step: 0.1483.. Train loss: 0.0680.. Train accuracy: 0.9763.. Top-3 train accuracy: 0.9992.. Test loss: 0.4342.. Test accuracy: 0.8983.. Top-3 test accuracy: 0.9829\n",
      "Epoch 48/100.. Time per epoch: 57.8841.. Average time per step: 0.1480.. Train loss: 0.0618.. Train accuracy: 0.9782.. Top-3 train accuracy: 0.9994.. Test loss: 0.4199.. Test accuracy: 0.8967.. Top-3 test accuracy: 0.9866\n",
      "Epoch 49/100.. Time per epoch: 58.0302.. Average time per step: 0.1484.. Train loss: 0.0571.. Train accuracy: 0.9801.. Top-3 train accuracy: 0.9996.. Test loss: 0.4471.. Test accuracy: 0.8951.. Top-3 test accuracy: 0.9845\n",
      "Epoch 50/100.. Time per epoch: 57.8835.. Average time per step: 0.1480.. Train loss: 0.0586.. Train accuracy: 0.9793.. Top-3 train accuracy: 0.9992.. Test loss: 0.4552.. Test accuracy: 0.8962.. Top-3 test accuracy: 0.9838\n",
      "Epoch 51/100.. Time per epoch: 58.1047.. Average time per step: 0.1486.. Train loss: 0.0560.. Train accuracy: 0.9799.. Top-3 train accuracy: 0.9993.. Test loss: 0.4810.. Test accuracy: 0.8934.. Top-3 test accuracy: 0.9831\n",
      "Epoch 52/100.. Time per epoch: 57.9741.. Average time per step: 0.1483.. Train loss: 0.0541.. Train accuracy: 0.9812.. Top-3 train accuracy: 0.9994.. Test loss: 0.5113.. Test accuracy: 0.8890.. Top-3 test accuracy: 0.9824\n",
      "Epoch 53/100.. Time per epoch: 57.9915.. Average time per step: 0.1483.. Train loss: 0.0553.. Train accuracy: 0.9809.. Top-3 train accuracy: 0.9993.. Test loss: 0.4760.. Test accuracy: 0.8953.. Top-3 test accuracy: 0.9833\n",
      "Epoch 54/100.. Time per epoch: 58.2543.. Average time per step: 0.1490.. Train loss: 0.0533.. Train accuracy: 0.9806.. Top-3 train accuracy: 0.9995.. Test loss: 0.4760.. Test accuracy: 0.8912.. Top-3 test accuracy: 0.9816\n",
      "Epoch 55/100.. Time per epoch: 58.2874.. Average time per step: 0.1491.. Train loss: 0.0488.. Train accuracy: 0.9829.. Top-3 train accuracy: 0.9996.. Test loss: 0.4730.. Test accuracy: 0.8951.. Top-3 test accuracy: 0.9839\n",
      "Epoch 56/100.. Time per epoch: 58.4716.. Average time per step: 0.1495.. Train loss: 0.0482.. Train accuracy: 0.9832.. Top-3 train accuracy: 0.9994.. Test loss: 0.4794.. Test accuracy: 0.8935.. Top-3 test accuracy: 0.9835\n",
      "Epoch 57/100.. Time per epoch: 58.3004.. Average time per step: 0.1491.. Train loss: 0.0497.. Train accuracy: 0.9823.. Top-3 train accuracy: 0.9995.. Test loss: 0.4655.. Test accuracy: 0.8949.. Top-3 test accuracy: 0.9835\n",
      "Epoch 58/100.. Time per epoch: 58.3081.. Average time per step: 0.1491.. Train loss: 0.0492.. Train accuracy: 0.9828.. Top-3 train accuracy: 0.9996.. Test loss: 0.4524.. Test accuracy: 0.8982.. Top-3 test accuracy: 0.9835\n",
      "Epoch 59/100.. Time per epoch: 58.1601.. Average time per step: 0.1487.. Train loss: 0.0422.. Train accuracy: 0.9849.. Top-3 train accuracy: 0.9996.. Test loss: 0.4793.. Test accuracy: 0.8948.. Top-3 test accuracy: 0.9840\n",
      "Epoch 60/100.. Time per epoch: 58.6814.. Average time per step: 0.1501.. Train loss: 0.0453.. Train accuracy: 0.9842.. Top-3 train accuracy: 0.9996.. Test loss: 0.4736.. Test accuracy: 0.8952.. Top-3 test accuracy: 0.9848\n",
      "Epoch 61/100.. Time per epoch: 58.3997.. Average time per step: 0.1494.. Train loss: 0.0438.. Train accuracy: 0.9847.. Top-3 train accuracy: 0.9995.. Test loss: 0.4581.. Test accuracy: 0.8949.. Top-3 test accuracy: 0.9855\n",
      "Epoch 62/100.. Time per epoch: 58.2919.. Average time per step: 0.1491.. Train loss: 0.0433.. Train accuracy: 0.9849.. Top-3 train accuracy: 0.9996.. Test loss: 0.4634.. Test accuracy: 0.8973.. Top-3 test accuracy: 0.9843\n",
      "Epoch 63/100.. Time per epoch: 58.3452.. Average time per step: 0.1492.. Train loss: 0.0396.. Train accuracy: 0.9865.. Top-3 train accuracy: 0.9998.. Test loss: 0.4982.. Test accuracy: 0.8921.. Top-3 test accuracy: 0.9834\n",
      "Epoch 64/100.. Time per epoch: 59.3747.. Average time per step: 0.1519.. Train loss: 0.0429.. Train accuracy: 0.9846.. Top-3 train accuracy: 0.9998.. Test loss: 0.4714.. Test accuracy: 0.8915.. Top-3 test accuracy: 0.9828\n",
      "Epoch 65/100.. Time per epoch: 59.2911.. Average time per step: 0.1516.. Train loss: 0.0418.. Train accuracy: 0.9853.. Top-3 train accuracy: 0.9996.. Test loss: 0.4850.. Test accuracy: 0.8938.. Top-3 test accuracy: 0.9844\n",
      "Epoch 66/100.. Time per epoch: 59.4622.. Average time per step: 0.1521.. Train loss: 0.0385.. Train accuracy: 0.9869.. Top-3 train accuracy: 0.9996.. Test loss: 0.4844.. Test accuracy: 0.8976.. Top-3 test accuracy: 0.9845\n",
      "Epoch 67/100.. Time per epoch: 59.2766.. Average time per step: 0.1516.. Train loss: 0.0393.. Train accuracy: 0.9865.. Top-3 train accuracy: 0.9997.. Test loss: 0.4763.. Test accuracy: 0.9004.. Top-3 test accuracy: 0.9852\n",
      "Epoch 68/100.. Time per epoch: 59.3961.. Average time per step: 0.1519.. Train loss: 0.0372.. Train accuracy: 0.9875.. Top-3 train accuracy: 0.9998.. Test loss: 0.5198.. Test accuracy: 0.8917.. Top-3 test accuracy: 0.9815\n",
      "Epoch 69/100.. Time per epoch: 59.0566.. Average time per step: 0.1510.. Train loss: 0.0383.. Train accuracy: 0.9864.. Top-3 train accuracy: 0.9996.. Test loss: 0.4918.. Test accuracy: 0.8919.. Top-3 test accuracy: 0.9842\n",
      "Epoch 70/100.. Time per epoch: 58.8006.. Average time per step: 0.1504.. Train loss: 0.0380.. Train accuracy: 0.9868.. Top-3 train accuracy: 0.9998.. Test loss: 0.5179.. Test accuracy: 0.8926.. Top-3 test accuracy: 0.9837\n",
      "Epoch 71/100.. Time per epoch: 58.9499.. Average time per step: 0.1508.. Train loss: 0.0367.. Train accuracy: 0.9876.. Top-3 train accuracy: 0.9996.. Test loss: 0.4880.. Test accuracy: 0.8926.. Top-3 test accuracy: 0.9831\n",
      "Epoch 72/100.. Time per epoch: 58.7406.. Average time per step: 0.1502.. Train loss: 0.0354.. Train accuracy: 0.9882.. Top-3 train accuracy: 0.9995.. Test loss: 0.4815.. Test accuracy: 0.8993.. Top-3 test accuracy: 0.9832\n",
      "Epoch 73/100.. Time per epoch: 58.9139.. Average time per step: 0.1507.. Train loss: 0.0333.. Train accuracy: 0.9882.. Top-3 train accuracy: 0.9998.. Test loss: 0.4841.. Test accuracy: 0.8973.. Top-3 test accuracy: 0.9825\n",
      "Epoch 74/100.. Time per epoch: 59.0054.. Average time per step: 0.1509.. Train loss: 0.0346.. Train accuracy: 0.9881.. Top-3 train accuracy: 0.9998.. Test loss: 0.5128.. Test accuracy: 0.8955.. Top-3 test accuracy: 0.9825\n",
      "Epoch 75/100.. Time per epoch: 59.1839.. Average time per step: 0.1514.. Train loss: 0.0329.. Train accuracy: 0.9884.. Top-3 train accuracy: 0.9998.. Test loss: 0.5019.. Test accuracy: 0.9021.. Top-3 test accuracy: 0.9849\n",
      "Epoch 76/100.. Time per epoch: 57.9126.. Average time per step: 0.1481.. Train loss: 0.0318.. Train accuracy: 0.9892.. Top-3 train accuracy: 0.9999.. Test loss: 0.5108.. Test accuracy: 0.8955.. Top-3 test accuracy: 0.9832\n",
      "Epoch 77/100.. Time per epoch: 58.4083.. Average time per step: 0.1494.. Train loss: 0.0311.. Train accuracy: 0.9895.. Top-3 train accuracy: 0.9998.. Test loss: 0.4937.. Test accuracy: 0.8987.. Top-3 test accuracy: 0.9843\n",
      "Epoch 78/100.. Time per epoch: 58.2263.. Average time per step: 0.1489.. Train loss: 0.0337.. Train accuracy: 0.9883.. Top-3 train accuracy: 0.9997.. Test loss: 0.4887.. Test accuracy: 0.8967.. Top-3 test accuracy: 0.9823\n",
      "Epoch 79/100.. Time per epoch: 57.9680.. Average time per step: 0.1483.. Train loss: 0.0348.. Train accuracy: 0.9882.. Top-3 train accuracy: 0.9998.. Test loss: 0.4860.. Test accuracy: 0.9002.. Top-3 test accuracy: 0.9838\n",
      "Epoch 80/100.. Time per epoch: 58.1310.. Average time per step: 0.1487.. Train loss: 0.0305.. Train accuracy: 0.9897.. Top-3 train accuracy: 0.9998.. Test loss: 0.4966.. Test accuracy: 0.8963.. Top-3 test accuracy: 0.9841\n",
      "Epoch 81/100.. Time per epoch: 58.1362.. Average time per step: 0.1487.. Train loss: 0.0308.. Train accuracy: 0.9897.. Top-3 train accuracy: 0.9998.. Test loss: 0.4912.. Test accuracy: 0.8995.. Top-3 test accuracy: 0.9860\n",
      "Epoch 82/100.. Time per epoch: 57.8786.. Average time per step: 0.1480.. Train loss: 0.0280.. Train accuracy: 0.9898.. Top-3 train accuracy: 0.9999.. Test loss: 0.4977.. Test accuracy: 0.8957.. Top-3 test accuracy: 0.9846\n",
      "Epoch 83/100.. Time per epoch: 57.4329.. Average time per step: 0.1469.. Train loss: 0.0280.. Train accuracy: 0.9903.. Top-3 train accuracy: 0.9998.. Test loss: 0.5081.. Test accuracy: 0.8971.. Top-3 test accuracy: 0.9838\n",
      "Epoch 84/100.. Time per epoch: 57.7010.. Average time per step: 0.1476.. Train loss: 0.0336.. Train accuracy: 0.9882.. Top-3 train accuracy: 0.9999.. Test loss: 0.5063.. Test accuracy: 0.8948.. Top-3 test accuracy: 0.9842\n",
      "Epoch 85/100.. Time per epoch: 57.5576.. Average time per step: 0.1472.. Train loss: 0.0261.. Train accuracy: 0.9910.. Top-3 train accuracy: 0.9999.. Test loss: 0.4800.. Test accuracy: 0.8961.. Top-3 test accuracy: 0.9851\n",
      "Epoch 86/100.. Time per epoch: 57.5026.. Average time per step: 0.1471.. Train loss: 0.0275.. Train accuracy: 0.9907.. Top-3 train accuracy: 0.9998.. Test loss: 0.4948.. Test accuracy: 0.8989.. Top-3 test accuracy: 0.9844\n",
      "Epoch 87/100.. Time per epoch: 57.3774.. Average time per step: 0.1467.. Train loss: 0.0282.. Train accuracy: 0.9901.. Top-3 train accuracy: 0.9998.. Test loss: 0.4867.. Test accuracy: 0.8998.. Top-3 test accuracy: 0.9848\n",
      "Epoch 88/100.. Time per epoch: 57.2970.. Average time per step: 0.1465.. Train loss: 0.0262.. Train accuracy: 0.9907.. Top-3 train accuracy: 0.9998.. Test loss: 0.4963.. Test accuracy: 0.9001.. Top-3 test accuracy: 0.9854\n",
      "Epoch 89/100.. Time per epoch: 57.3851.. Average time per step: 0.1468.. Train loss: 0.0258.. Train accuracy: 0.9908.. Top-3 train accuracy: 1.0000.. Test loss: 0.5331.. Test accuracy: 0.8952.. Top-3 test accuracy: 0.9836\n",
      "Epoch 90/100.. Time per epoch: 57.2572.. Average time per step: 0.1464.. Train loss: 0.0291.. Train accuracy: 0.9900.. Top-3 train accuracy: 0.9998.. Test loss: 0.5104.. Test accuracy: 0.8941.. Top-3 test accuracy: 0.9841\n",
      "Epoch 91/100.. Time per epoch: 57.3480.. Average time per step: 0.1467.. Train loss: 0.0242.. Train accuracy: 0.9917.. Top-3 train accuracy: 0.9999.. Test loss: 0.4944.. Test accuracy: 0.9011.. Top-3 test accuracy: 0.9860\n",
      "Epoch 92/100.. Time per epoch: 57.3575.. Average time per step: 0.1467.. Train loss: 0.0241.. Train accuracy: 0.9916.. Top-3 train accuracy: 0.9998.. Test loss: 0.5331.. Test accuracy: 0.8958.. Top-3 test accuracy: 0.9850\n",
      "Epoch 93/100.. Time per epoch: 57.3366.. Average time per step: 0.1466.. Train loss: 0.0255.. Train accuracy: 0.9909.. Top-3 train accuracy: 0.9998.. Test loss: 0.4910.. Test accuracy: 0.9008.. Top-3 test accuracy: 0.9836\n",
      "Epoch 94/100.. Time per epoch: 57.6525.. Average time per step: 0.1474.. Train loss: 0.0250.. Train accuracy: 0.9913.. Top-3 train accuracy: 0.9999.. Test loss: 0.5234.. Test accuracy: 0.8969.. Top-3 test accuracy: 0.9846\n",
      "Epoch 95/100.. Time per epoch: 57.6418.. Average time per step: 0.1474.. Train loss: 0.0225.. Train accuracy: 0.9923.. Top-3 train accuracy: 0.9999.. Test loss: 0.5251.. Test accuracy: 0.8980.. Top-3 test accuracy: 0.9846\n",
      "Epoch 96/100.. Time per epoch: 57.6497.. Average time per step: 0.1474.. Train loss: 0.0272.. Train accuracy: 0.9908.. Top-3 train accuracy: 0.9999.. Test loss: 0.5212.. Test accuracy: 0.8975.. Top-3 test accuracy: 0.9840\n",
      "Epoch 97/100.. Time per epoch: 57.6144.. Average time per step: 0.1474.. Train loss: 0.0236.. Train accuracy: 0.9918.. Top-3 train accuracy: 0.9999.. Test loss: 0.4954.. Test accuracy: 0.9033.. Top-3 test accuracy: 0.9835\n",
      "Epoch 98/100.. Time per epoch: 57.7673.. Average time per step: 0.1477.. Train loss: 0.0249.. Train accuracy: 0.9916.. Top-3 train accuracy: 0.9998.. Test loss: 0.4987.. Test accuracy: 0.9027.. Top-3 test accuracy: 0.9850\n",
      "Epoch 99/100.. Time per epoch: 57.6581.. Average time per step: 0.1475.. Train loss: 0.0217.. Train accuracy: 0.9923.. Top-3 train accuracy: 0.9999.. Test loss: 0.5139.. Test accuracy: 0.8985.. Top-3 test accuracy: 0.9845\n",
      "Epoch 100/100.. Time per epoch: 57.5586.. Average time per step: 0.1472.. Train loss: 0.0233.. Train accuracy: 0.9921.. Top-3 train accuracy: 0.9998.. Test loss: 0.5443.. Test accuracy: 0.8991.. Top-3 test accuracy: 0.9841\n"
     ]
    }
   ],
   "source": [
    "#train the model\n",
    "model.to(device)\n",
    "\n",
    "steps = 0\n",
    "running_loss = 0\n",
    "for epoch in range(epochs):\n",
    "    \n",
    "    since = time.time()\n",
    "    \n",
    "    train_accuracy = 0\n",
    "    top3_train_accuracy = 0 \n",
    "    for inputs, labels in trainloader:\n",
    "        steps += 1\n",
    "        # Move input and label tensors to the default device\n",
    "        inputs, labels = inputs.to(device), labels.to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        logps = model.forward(inputs)\n",
    "        loss = criterion(logps, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        running_loss += loss.item()\n",
    "        \n",
    "        # calculate train top-1 accuracy\n",
    "        ps = torch.exp(logps)\n",
    "        top_p, top_class = ps.topk(1, dim=1)\n",
    "        equals = top_class == labels.view(*top_class.shape)\n",
    "        train_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "        \n",
    "        # Calculate train top-3 accuracy\n",
    "        np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "        target_numpy = labels.cpu().numpy()\n",
    "        top3_train_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "        \n",
    "    time_elapsed = time.time() - since\n",
    "    \n",
    "    test_loss = 0\n",
    "    test_accuracy = 0\n",
    "    top3_test_accuracy = 0\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for inputs, labels in testloader:\n",
    "            inputs, labels = inputs.to(device), labels.to(device)\n",
    "            logps = model.forward(inputs)\n",
    "            batch_loss = criterion(logps, labels)\n",
    "\n",
    "            test_loss += batch_loss.item()\n",
    "\n",
    "            # Calculate test top-1 accuracy\n",
    "            ps = torch.exp(logps)\n",
    "            top_p, top_class = ps.topk(1, dim=1)\n",
    "            equals = top_class == labels.view(*top_class.shape)\n",
    "            test_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "            \n",
    "            # Calculate test top-3 accuracy\n",
    "            np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "            target_numpy = labels.cpu().numpy()\n",
    "            top3_test_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "\n",
    "    print(f\"Epoch {epoch+1}/{epochs}.. \"\n",
    "          f\"Time per epoch: {time_elapsed:.4f}.. \"\n",
    "          f\"Average time per step: {time_elapsed/len(trainloader):.4f}.. \"\n",
    "          f\"Train loss: {running_loss/len(trainloader):.4f}.. \"\n",
    "          f\"Train accuracy: {train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Top-3 train accuracy: {top3_train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Test loss: {test_loss/len(testloader):.4f}.. \"\n",
    "          f\"Test accuracy: {test_accuracy/len(testloader):.4f}.. \"\n",
    "          f\"Top-3 test accuracy: {top3_test_accuracy/len(testloader):.4f}\")\n",
    "\n",
    "    train_stats = train_stats.append({'Epoch': epoch, 'Time per epoch':time_elapsed, 'Avg time per step': time_elapsed/len(trainloader), 'Train loss' : running_loss/len(trainloader), 'Train accuracy': train_accuracy/len(trainloader), 'Train top-3 accuracy':top3_train_accuracy/len(trainloader),'Test loss' : test_loss/len(testloader), 'Test accuracy': test_accuracy/len(testloader), 'Test top-3 accuracy':top3_test_accuracy/len(testloader)}, ignore_index=True)\n",
    "\n",
    "    running_loss = 0\n",
    "    model.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats.to_csv('train_log_SENet18_Mish.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
